const fs = require("fs");
const path = require("path");
const scrapy = require("node-scrapy");

//避免重复抓取
let urlset = new Set();

const main_menu = {
    titles: {
        selector: ".head-2a-list-b>a",
        get: "text"
    },
    urls: {
        selector: ".head-2a-list-b>a",
        get: "href"
    }
};
//子页模型
const sub_menu = {
    //子菜单项
    items: {
        titles: {
            selector: ".k_list-1a-1a",
            get: "text"
        },
        urls: {
            selector: ".k_list-1a-1b>a",
            get: "href"
        }
    },
    //排行榜
    tops: {
        titles: {
            selector: ".k_movie-1b-b1-title>a",
            get: "title"
        },
        urls: {
            selector: ".k_movie-1b-b1-title>a",
            get: "href"
        }
    }
}

//在第一页解析所有页面url的模型
const pages_model = {
    //列表描述
    desc: ".k_pape>span",
    //每页地址
    pages: {
        selector: ".k_pape>select>option:first-child ~ option",
        get: "value"
    }
}

const list_model = {
    imgs: {
        selector: "#k_upnew-1d-img>a>img",
        get: "src"
    },
    titles: {
        selector: "#k_list-lb-2-a>a",
        get: "title",
        transform:function(){
            return this.toString().replace(/[\/"\\:*?><|]/g," ");
        }
    },
    urls: {
        selector: "#k_list-lb-2-a>a",
        get: "href"
    }
}

const content_model = {
    bt_url:{
        selector:".k_jianjie-3a-5 li:last-child>a",
        get:"href"
    },
    emule_url: {
        selector: ".k_jianjie-3a-5down",
        get: "text"
    },
    img:{
        selector:"#k_jianjie-2b>a>img",
        get:"src"
    },
    frames:{
        selector:".content.vimg>p>img",
        get:"src",
        unique:false
    },
    title:{
        selector:".k_jianjie-3a-1 li",
        transform:function(){
            return this.toString().replace(/[\/"\\:*?><|]/g," ");
        }
    }
}

const savedir="./save";
if(!fs.existsSync(savedir))
{
    fs.mkdirSync(savedir);
}


//工具函数
let paths=[];
function getpath()
{
    return path.join(savedir,...paths);
}
function getfilename(fname)
{
    return path.join(getpath(),fname);
}
/**
 * 保证paths表示的目录存在
 */
function ensure_paths()
{
    if(!fs.existsSync(getpath()))
    {
        fs.mkdirSync(getpath());
    }
}
//确保子目录存在
function ensure_dir(dir)
{

    let p=path.join(getpath(),dir);
    if(!fs.existsSync(p))
    {
        fs.mkdirSync(p);
    }
}
function enter_dir(name)
{
    //防止错误字符
    name=name.replace(/[\/"\\:*?><|]/g," ");

    paths.push(name);
    ensure_paths();

}

function leave_dir()
{
    paths.pop();
}

//下载部分
const req=require("request");
let fail_num=1;
function download(url,fpath)
{
    try
    {
        req.get(url,(err,res,body)=>{
            if(err==null);
                //res.pipe(fs.createWriteStream(fpath));
            else
            {
                console.log(`${fail_num++}个文件下载失败!`);
            }
        }).pipe(fs.createWriteStream(fpath));
    }
    catch(e)
    {
        console.log(`${fail_num++}个文件下载失败!`);
    }
    
}

//同步化
async function scrape_async(url,model)
{
    return new Promise((resolve,reject)=>{
        scrapy.scrape(url,model,async (err,data)=>{
            resolve(data);
        })
    })
}

function writefile(file,lst)
{
    let str="";
        lst.forEach(element => {
           str+=`${element}\n`; 
        });
    fs.writeFileSync(file,str);
}

function sleep(milliSeconds) { 
    var startTime = new Date().getTime(); 
    while (new Date().getTime() < startTime + milliSeconds);
 };


///抓取部分
const baseurl = "http://www.ririri.info";

let invalid_lists = 0;

async function parse_main(data) {
    //data为大类列表（电影 图片等）
    //加载子页
    //保存每个大类的elist
    let elist=[];
    //本为data.titles.length 但这里只抓电影
    for(let i=0;i<1;++i)
    {
        let title=data.titles[i];
        let url=data.urls[i];
        if(url==null) continue;
        enter_dir(title);
        let ret=await parse_sub(await scrape_async(baseurl + url, sub_menu));
        //保存 按大类
        writefile(getfilename("emules.txt"),ret);
        leave_dir();
        elist=elist.concat(ret);
        
    }
    return elist;
    
}



let start=0,end=-1;
async function parse_sub(data) {
    //进入子目录
    //data为子类列表（欧美 亚洲等）
    //这里保存每个子类的elist
    let elist=[];
    for(let i=0;i<data.items.titles.length;++i)
    {
        console.log(`${i}. ${data.items.titles[i]}`);

    }
    if(end==-1) end=data.items.titles.length;
    for(let i=start;i<end;++i)
    {
        let title=data.items.titles[i];
        let url=data.items.urls[i];
        if(url==null) continue;
        enter_dir(title);
        //返回一个子类的链接列表
        let ret=await parse_pages(await scrape_async(baseurl + url, pages_model));
        //保存到文件(每个子类一个文件)
        writefile(getfilename("emules.txt"),ret);
        leave_dir();
        elist=elist.concat(ret);
        
    }
    return elist;
}

async function parse_pages(data) {
    //加载每一页 data为页列表 即每个子类中有n个list页
    let pages = data.pages;
    //为所有page（即一个子类）的链接
    let elist=[];
    for (let url of pages) {
        if(url==null) continue;
        let ret=await parse_list(await scrape_async(baseurl + url, list_model));
        elist=elist.concat(ret);
    }
    return elist;
}



async function parse_list(data) {
    //此时data为列表model
    //尝试解析列表中的每一内容页
    //这里记录一个总emule链接表
    if(data==null) return [];
    let elist=[];
    if (data.urls != null)
    {
        
        for(let i=0;i<data.titles.length;++i)
        {
            let title=data.titles[i];
            let url=data.urls[i];
            if(url==null) continue;
            //此处避免重复加载
            if(fs.existsSync(getfilename(title)))
            {
                //直接从目录读取emule地址
                enter_dir(title);
                let str;
                try
                {
                    str=fs.readFileSync(getfilename("emule.txt")).toString();
                }
                catch(e)
                {
                    console.log(`读取emule.txt失败:${title}`);
                    leave_dir();
                    continue;
                }
                
                str=str.trim();
                elist.push(str);
                console.log(`跳过${title}`);
                leave_dir();
                continue;
            }
            let imageurl=data.imgs[i];
            //此处不进入目录 由cont解析器决定是否创建目录
            //enter_dir(title);
            //此处内容解析函数发挥的事emule链接
            let eurl=await parse_cont(await scrape_async(baseurl + url, content_model));
            elist.push(eurl);
        }
    }
    else
        console.log(`发现无效列表页:${++invalid_lists}`);
    //此处返回此列表页的所有emule链接
    return elist;
}

function parse_cont(data) {
    //进入相应目录
    if(data==null) return "";
    //如果已经存在指定目录则跳过
    if(fs.existsSync(getfilename(data.title))) {
        console.log(`跳过${data.title}`);
        return data.emule_url!=null? data.emule_url:"";
    }
    enter_dir(data.title);
    //保存封面
    if(data.img!=null)
        download(data.img,getfilename("title.jpg"));
    if(data.frames!=null)
        data.frames.forEach((ele,idx)=>{
            if(ele!=null)
                download(ele,getfilename(`${idx}.jpg`));
        });
    //保存bt种子 这里由于页面原因 bturl会获取两个url 第二个为bt种子链接
    if(data.bt_url!=null)
        download(data.bt_url[1],getfilename("bt.torrent"));
    //保存电驴链接
    if(data.emule_url!=null)
        fs.writeFileSync(getfilename("emule.txt"),data.emule_url);
    //文件操作完成 退出目录
    leave_dir();
    //返回此页的电驴链接
    console.log(data.emule_url);
    sleep(500);
    return data.emule_url;
}

let run=async ()=>{await parse_main(await scrape_async(baseurl, main_menu));}
run();